# 2012/10/12
# This program is written for generating features between queries and documents.
# Let's do the complex features, man.
from __future__ import division
import os

def computeComplexFeaturesSupportedData(doc_words,queryVariationID,queryVariationString,outputFileHandler,cf_dict_in_function,df_dict_in_function,queryID,trecID):
    # print doc_words
    
    print "    ---> Processing:",queryVariationID,queryVariationString
    indicesList = []
    queryVariationList = queryVariationString.split("_")
    for queryTerm in queryVariationList:
        currentIndices = [i for i, x in enumerate(doc_words) if x == queryTerm]
        indicesList.append(currentIndices)
    
    #print "raw indices"

    
    M_Constant_32 = 32
    M_Constant_8 = 8
    M_Constant_4 = 4
    M_Constant_1 = 1
    
    N_Constant_32 = 32
    N_Constant_8 = 8
    N_Constant_4 = 4
    N_Constant_1 = 1
    
    tf_M_1 = 0
    tf_M_4 = 0
    tf_M_8 = 0
    tf_M_32 = 0
    
    tf_N_1 = 0
    tf_N_4 = 0
    tf_N_8 = 0
    tf_N_32 = 0
    
    
    
    # Let's do the M computation
    # need to assign the head indices
    # a dict structure for final judge
    tempImportantPositionsList_M32 = []
    tempImportantPositionsList_M8 = []
    tempImportantPositionsList_M4 = []
    tempImportantPositionsList_M1 = []
    
    indicesConditionDict_M32 = {}
    indicesConditionDict_M8 = {}
    indicesConditionDict_M4 = {}
    indicesConditionDict_M1 = {}
    
    for i in range(0,len(indicesList)):
        indicesConditionDict_M32[i] = False
        indicesConditionDict_M8[i] = False
        indicesConditionDict_M4[i] = False
        indicesConditionDict_M1[i] = False
    # print "old indicesConditionDict_M32:",indicesConditionDict_M32
    # set the head indices to be True
    indicesConditionDict_M32[0] = True
    indicesConditionDict_M8[0] = True
    indicesConditionDict_M4[0] = True
    indicesConditionDict_M1[0] = True
    
    # assignment loop
    for indexNumberFromIndex1 in indicesList[0]:
        
        tempImportantPositionsList_M32.append( (indexNumberFromIndex1,0) )
        tempImportantPositionsList_M8.append( (indexNumberFromIndex1,0) )
        tempImportantPositionsList_M4.append( (indexNumberFromIndex1,0) )
        tempImportantPositionsList_M1.append( (indexNumberFromIndex1,0) )
        
        for index,currentIndices in enumerate(indicesList[1:]):
            # for the other indices
            for currentIndicesPositionNumber in currentIndices:
                # core judge statements
                if currentIndicesPositionNumber >= indexNumberFromIndex1 and currentIndicesPositionNumber <= indexNumberFromIndex1 + M_Constant_1 + 1:
                    indicesConditionDict_M1[index+1] = True
                    tempImportantPositionsList_M1.append( (currentIndicesPositionNumber,index+1) )
                
                if currentIndicesPositionNumber >= indexNumberFromIndex1 and currentIndicesPositionNumber <= indexNumberFromIndex1 + M_Constant_4 + 1:
                    indicesConditionDict_M4[index+1] = True
                    tempImportantPositionsList_M4.append( (currentIndicesPositionNumber,index+1) )
                
                if currentIndicesPositionNumber >= indexNumberFromIndex1 and currentIndicesPositionNumber <= indexNumberFromIndex1 + M_Constant_8 + 1:
                    indicesConditionDict_M8[index+1] = True
                    tempImportantPositionsList_M8.append( (currentIndicesPositionNumber,index+1) )
                
                if currentIndicesPositionNumber >= indexNumberFromIndex1 and currentIndicesPositionNumber <= indexNumberFromIndex1 + M_Constant_32 + 1:
                    indicesConditionDict_M32[index+1] = True
                    tempImportantPositionsList_M32.append( (currentIndicesPositionNumber,index+1) )
                    break
              
        # judge loop
        allConditionSafisfiedFlag_M32 = True
        allConditionSafisfiedFlag_M8 = True
        allConditionSafisfiedFlag_M4 = True
        allConditionSafisfiedFlag_M1 = True
        

        for i in indicesConditionDict_M32:
            if indicesConditionDict_M32[i] == False:
                allConditionSafisfiedFlag_M32 =False

        for i in indicesConditionDict_M8:
            if indicesConditionDict_M8[i] == False:
                allConditionSafisfiedFlag_M8 =False

        for i in indicesConditionDict_M4:
            if indicesConditionDict_M4[i] == False:
                allConditionSafisfiedFlag_M4 =False

        for i in indicesConditionDict_M1:
            if indicesConditionDict_M1[i] == False:
                allConditionSafisfiedFlag_M1 =False


        if allConditionSafisfiedFlag_M32:
            #print 
            #print indicesConditionDict_M32
            #print tempImportantPositionsList_M32
            
            minPosition = 999999
            maxPosition = 0
            outputLine = ""
            for currentTuple in tempImportantPositionsList_M32:
                (currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "
                
                if currentPosition < minPosition:
                    minPosition = currentPosition
                
                if currentPosition > maxPosition:
                    maxPosition = currentPosition
            #print "minPosition:",minPosition
            #print "maxPosition:",maxPosition
            #print outputLine
            #print doc_words[minPosition:maxPosition+1]
            tf_M_32 += 1

        if allConditionSafisfiedFlag_M8:
            #print 
            #print indicesConditionDict_M8
            #print tempImportantPositionsList_M8
            
            minPosition = 999999
            maxPosition = 0
            outputLine = ""
            for currentTuple in tempImportantPositionsList_M8:
                (currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "
                
                if currentPosition < minPosition:
                    minPosition = currentPosition
                
                if currentPosition > maxPosition:
                    maxPosition = currentPosition
            #print "minPosition:",minPosition
            #print "maxPosition:",maxPosition
            #print outputLine
            #print doc_words[minPosition:maxPosition+1]
            tf_M_8 += 1

        if allConditionSafisfiedFlag_M4:
            #print 
            #print indicesConditionDict_M4
            #print tempImportantPositionsList_M4
            
            minPosition = 999999
            maxPosition = 0
            outputLine = ""
            for currentTuple in tempImportantPositionsList_M4:
                (currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "
                
                if currentPosition < minPosition:
                    minPosition = currentPosition
                
                if currentPosition > maxPosition:
                    maxPosition = currentPosition
            #print "minPosition:",minPosition
            #print "maxPosition:",maxPosition
            #print outputLine
            #print doc_words[minPosition:maxPosition+1]
            tf_M_4 += 1
        
        # for M1
        if allConditionSafisfiedFlag_M1:
            #print 
            #print indicesConditionDict_M1
            #print tempImportantPositionsList_M1
            
            minPosition = 999999
            maxPosition = 0
            outputLine = ""
            for currentTuple in tempImportantPositionsList_M1:
                (currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "
                
                if currentPosition < minPosition:
                    minPosition = currentPosition
                
                if currentPosition > maxPosition:
                    maxPosition = currentPosition
            #print "minPosition:",minPosition
            #print "maxPosition:",maxPosition
            #print outputLine
            #print doc_words[minPosition:maxPosition+1]
            tf_M_1 += 1            
            
                     
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_M32[i] = False
        indicesConditionDict_M32[0] = True
        del tempImportantPositionsList_M32[:]
        
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_M8[i] = False
        indicesConditionDict_M8[0] = True
        del tempImportantPositionsList_M8[:]
        
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_M4[i] = False
        indicesConditionDict_M4[0] = True
        del tempImportantPositionsList_M4[:]
        
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_M1[i] = False
        indicesConditionDict_M1[0] = True
        del tempImportantPositionsList_M1[:]
        
        
#######################################################
    # updated by Wei: 2012/10/19. I assume there are NO problems for the M related computation
    # Now, I have identified the problem. It is because the N value is NOT computed well.
    # Let's do the N computation
    # do NOT need to assign the head indices
    # a dict structure for final judge
    tempImportantPositionsList_N32 = []
    tempImportantPositionsList_N8 = []
    tempImportantPositionsList_N4 = []
    tempImportantPositionsList_N1 = []
    
    indicesConditionDict_N32 = {}
    indicesConditionDict_N8 = {}
    indicesConditionDict_N4 = {}
    indicesConditionDict_N1 = {}
    
    # init condition requirement to be all false
    for i in range(0,len(indicesList)):
        indicesConditionDict_N32[i] = False
        indicesConditionDict_N8[i] = False
        indicesConditionDict_N4[i] = False
        indicesConditionDict_N1[i] = False
    
    
    # do NOT need to do this for N
    # indicesConditionDict_N32[0] = True
    # indicesConditionDict_N8[0] = True
    # indicesConditionDict_N4[0] = True
    # indicesConditionDict_N1[0] = True
    
    # assignment loop
    for indexNumberFromDoc,currentWord in enumerate(doc_words):
        # The variable currentWord is currently NOT used.
        # print "indexNumberFromDoc:",indexNumberFromDoc
        for index,currentIndices in enumerate(indicesList):
            # for the other indices
            for currentIndicesPositionNumber in currentIndices:
                # core judge statements
                # need to maintain the window size to be 1
                
                # in debug mode
                # if currentIndicesPositionNumber >= indexNumberFromDoc and currentIndicesPositionNumber <= indexNumberFromDoc + N_Constant_1:
                
                # in production mode
                if currentIndicesPositionNumber >= indexNumberFromDoc and currentIndicesPositionNumber <= indexNumberFromDoc + N_Constant_1 - 1:
                    #print "enter mark 1"
                    indicesConditionDict_N1[index] = True
                    tempImportantPositionsList_N1.append( (indexNumberFromDoc,currentIndicesPositionNumber,index) )
                
                # need to maintain the window size to be 4
                if currentIndicesPositionNumber >= indexNumberFromDoc and currentIndicesPositionNumber <= indexNumberFromDoc + N_Constant_4 - 1:
                    #print "enter mark 4"
                    indicesConditionDict_N4[index] = True
                    tempImportantPositionsList_N4.append( (indexNumberFromDoc,currentIndicesPositionNumber,index) )
                
                # need to maintain the window size to be 8
                if currentIndicesPositionNumber >= indexNumberFromDoc and currentIndicesPositionNumber <= indexNumberFromDoc + N_Constant_8 - 1:
                    # print "enter mark 8"
                    indicesConditionDict_N8[index] = True
                    tempImportantPositionsList_N8.append( (indexNumberFromDoc,currentIndicesPositionNumber,index) )
                
                # need to maintain the window size to be 32
                if currentIndicesPositionNumber >= indexNumberFromDoc and currentIndicesPositionNumber <= indexNumberFromDoc + N_Constant_32 - 1:
                    #print "enter mark 32"
                    indicesConditionDict_N32[index] = True
                    tempImportantPositionsList_N32.append( (indexNumberFromDoc,currentIndicesPositionNumber,index) )
                    break
              
        # judge loop
        allConditionSafisfiedFlag_N32 = True
        allConditionSafisfiedFlag_N8 = True
        allConditionSafisfiedFlag_N4 = True
        allConditionSafisfiedFlag_N1 = True
        

        for i in indicesConditionDict_N32:
            if indicesConditionDict_N32[i] == False:
                allConditionSafisfiedFlag_N32 =False

        for i in indicesConditionDict_N8:
            if indicesConditionDict_N8[i] == False:
                allConditionSafisfiedFlag_N8 =False

        for i in indicesConditionDict_N4:
            if indicesConditionDict_N4[i] == False:
                allConditionSafisfiedFlag_N4 =False

        for i in indicesConditionDict_N1:
            if indicesConditionDict_N1[i] == False:
                allConditionSafisfiedFlag_N1 =False


        if allConditionSafisfiedFlag_N32:
            #print 
            #print indicesConditionDict_N32
            #print tempImportantPositionsList_N32
            
            outputLine = ""
            for currentTuple in tempImportantPositionsList_N32:
                (current_doc_position,currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "
                
            #print outputLine
            #print doc_words[current_doc_position:current_doc_position + N_Constant_32]
            tf_N_32 += 1

        if allConditionSafisfiedFlag_N8:
            #print 
            #print indicesConditionDict_N8
            #print tempImportantPositionsList_N8
            
            outputLine = ""
            for currentTuple in tempImportantPositionsList_N8:
                (current_doc_position,currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "

            #print outputLine
            #print doc_words[current_doc_position:current_doc_position + N_Constant_8]
            tf_N_8 += 1

        if allConditionSafisfiedFlag_N4:
            #print 
            #print indicesConditionDict_N4
            #print tempImportantPositionsList_N4
            
            outputLine = ""
            for currentTuple in tempImportantPositionsList_N4:
                (current_doc_position,currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "
                
            #print outputLine
            #print doc_words[current_doc_position:current_doc_position + N_Constant_4]
            tf_N_4 += 1
        
        # for M1
        if allConditionSafisfiedFlag_N1:
            #print 
            #print indicesConditionDict_N1
            #print tempImportantPositionsList_N1
            
            outputLine = ""
            for currentTuple in tempImportantPositionsList_N1:
                (current_doc_position,currentPosition,_) = currentTuple
                
                outputLine += doc_words[currentPosition] + " "
                
            #print outputLine
            #print doc_words[current_doc_position:current_doc_position + N_Constant_1]
            tf_N_1 += 1            
            
                     
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_N32[i] = False
        del tempImportantPositionsList_N32[:]
        
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_N8[i] = False
        del tempImportantPositionsList_N8[:]
        
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_N4[i] = False
        del tempImportantPositionsList_N4[:]
        
        # reset the judge dict
        for i in range(0,len(indicesList)):
            indicesConditionDict_N1[i] = False
        del tempImportantPositionsList_N1[:]
        
        
        
        
        
        
        
#######################################################
    if tf_M_32 != 0:
        df_dict_in_function["df_M_32"] += 1
        
    if tf_M_8 != 0:
        df_dict_in_function["df_M_8"] += 1
        
    if tf_M_4 != 0:
        df_dict_in_function["df_M_4"] += 1
        
    if tf_M_1 != 0:
        df_dict_in_function["df_M_1"] += 1


    if tf_N_32 != 0:
        df_dict_in_function["df_N_32"] += 1
        
    if tf_N_8 != 0:
        df_dict_in_function["df_N_8"] += 1
        
    if tf_N_4 != 0:
        df_dict_in_function["df_N_4"] += 1
        
    if tf_N_1 != 0:
        df_dict_in_function["df_N_1"] += 1
        
    # debug    
    # print "tf_M_32:",tf_M_32
    # print "tf_M_8:",tf_M_8
    # print "tf_M_4:",tf_M_4
    # print "tf_M_1:",tf_M_1
    
    cf_dict_in_function["cf_M_32"] += tf_M_32
    cf_dict_in_function["cf_M_8"] += tf_M_8
    cf_dict_in_function["cf_M_4"] += tf_M_4
    cf_dict_in_function["cf_M_1"] += tf_M_1

    cf_dict_in_function["cf_N_32"] += tf_N_32
    cf_dict_in_function["cf_N_8"] += tf_N_8
    cf_dict_in_function["cf_N_4"] += tf_N_4
    cf_dict_in_function["cf_N_1"] += tf_N_1    
    
    # old version. 2012/10/19
    # outputFileHandler.write(queryID + " " + trecID + " " + str(queryVariationID) + " " + queryVariationString + " "+ str(tf_M_1) + " " + str(tf_M_4) + " " + str(tf_M_8) + " " + str(tf_M_32) + " "+ str(tf_N_1) + " " + str(tf_N_4) + " " + str(tf_N_8) + " " + str(tf_N_32) + "\n")
    
    # new version.
    outputFileHandler.write(queryID + " " + trecID + " " + str(queryVariationID) + " " + str(tf_M_1) + " " + str(tf_M_4) + " " + str(tf_M_8) + " " + str(tf_M_32) + " "+ str(tf_N_1) + " " + str(tf_N_4) + " " + str(tf_N_8) + " " + str(tf_N_32) + "\n")
    
    #print "        ","tf_M_1:",tf_M_1,"tf_M_4:",tf_M_4,"tf_M_8:",tf_M_8,"tf_M_32:",tf_M_32,"tf_N_1:",tf_N_1,"tf_N_4:",tf_N_4,"tf_N_8:",tf_N_8,"tf_N_32:",tf_N_32


print "main program begins,updated by Wei:2012/10/19"
print "ongoing1: change the N to N order(1,4,8,32) and N unordered(1,4,8,32)"
print "ongoing2: improve the N alg to generate more OK results"
# step0: init some variables

outputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-complexFeaturesSupportedFileForTf_dict.txt"
outputFileHandler = open(outputFileName,"w")


# step1
queryVariationTupleList = []
# step1: read the query variations
inputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-150Clueweb09Queries-variations.head.2.txt"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    # print line
    lineElements = line.strip().split(" ")
    
    queryID = lineElements[0].split("_")[0]
    attr = lineElements[0].split("_")[1]
    queryVariations = lineElements[1:]
    
    # print "queryID:",queryID
    # print "attr:",attr
    # print "queryVariations:",queryVariations
    
    queryVariationTuple = (queryID,attr,queryVariations)
    queryVariationTupleList.append(queryVariationTuple)
inputFileHandler.close()
print "# of queries:",len(queryVariationTupleList)

cf_dicts = {}
df_dicts = {}

# give each variant an ID and output to the disk
tempNumOfVariations = 0
tempOutputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-150Clueweb09Queries-variations-only.txt"
tempOutputFileHandler = open(tempOutputFileName,"w")

for queryVariationTuple in queryVariationTupleList:
    (queryID,attr,queryVariations) = queryVariationTuple
    # tempNumOfVariations += len(queryVariations)
    for currentVariant in queryVariations:
        tempOutputFileHandler.write(str(tempNumOfVariations) + ":" + currentVariant + "\n")
        tempNumOfVariations += 1
        
        current_cf_dict = {}
        current_cf_dict["cf_M_1"] = 0
        current_cf_dict["cf_M_4"] = 0
        current_cf_dict["cf_M_8"] = 0
        current_cf_dict["cf_M_32"] = 0
        
        current_cf_dict["cf_N_1"] = 0
        current_cf_dict["cf_N_4"] = 0
        current_cf_dict["cf_N_8"] = 0
        current_cf_dict["cf_N_32"] = 0
        
        current_df_dict = {}
        current_df_dict["df_M_1"] = 0
        current_df_dict["df_M_4"] = 0
        current_df_dict["df_M_8"] = 0
        current_df_dict["df_M_32"] = 0
        
        current_df_dict["df_N_1"] = 0
        current_df_dict["df_N_4"] = 0
        current_df_dict["df_N_8"] = 0
        current_df_dict["df_N_32"] = 0
        
        if currentVariant not in cf_dicts:
            cf_dicts[currentVariant] = current_cf_dict
        else:
            print "mark1"
            sys.exit(1)
        
        if currentVariant not in df_dicts:
            df_dicts[currentVariant] = current_df_dict
        else:
            print "mark2"
            sys.exit(1)
        
print "tempNumOfVariations:",tempNumOfVariations
tempOutputFileHandler.close()


# load back the variations and the variationID:

queryVarianceList = []
tempInputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-150Clueweb09Queries-variations-only.txt"
tempInputFileHandler = open(tempInputFileName,"r")
for line in tempInputFileHandler.readlines():
    queryVarianceID = line.strip().split(":")[0]
    queryVarianceContent = line.strip().split(":")[1]
    queryVarianceTuple = (queryVarianceID,queryVarianceContent)
    queryVarianceList.append(queryVarianceTuple)
tempInputFileHandler.close()

print "len(queryVarianceList):",len(queryVarianceList)




# step2: read the document, compute and output
pathBase = "/data4/team/weijiang/human_judge_web_pages_plain_text_only_words/en0000"
for dirname, dirnames, filenames in os.walk(pathBase):
    for index,filename in enumerate(filenames):
        trecID = filename.split("_")[0]
        if index == 100:
            break
        absolutePathForWebPageFileName = os.path.join(dirname, filename)
        # print index,absolutePathForWebPageFileName

        
        #################################################production begins...    
        print "--->",index,absolutePathForWebPageFileName
        documentFileHandler = open(absolutePathForWebPageFileName,"r")
        doc_content = documentFileHandler.read()
        doc_words = doc_content.strip().split(" ")
    
        for tempTestTuple in queryVarianceList:
            (queryVarianceID,queryVarianceContent) = tempTestTuple
            computeComplexFeaturesSupportedData(doc_words,queryVarianceID,queryVarianceContent,outputFileHandler,cf_dicts[queryVarianceContent],df_dicts[queryVarianceContent],queryID,trecID)
            #print "        cf_dicts[",queryVarianceContent,"]:",cf_dicts[queryVarianceContent]
            #print "        df_dicts[",queryVarianceContent,"]:",df_dicts[queryVarianceContent]
        
        print
        
        #################################################production ends.
        
                
        
        '''
        #################################################debug begins...
        
        debugDocumentList = []
        
        # debug use document1
        absolutePathForWebPageFileName = "/data4/team/weijiang/human_judge_web_pages_plain_text_only_words/en0000/clueweb09-en0000-15-17511_1006_plain_text.txt"
        queryID = "1"
        trecID = "clueweb09-en0000-15-17511"
        tempDebugDocumentTuple = (queryID,trecID,absolutePathForWebPageFileName)
        debugDocumentList.append(tempDebugDocumentTuple)
        
        
        # debug use document2
        # absolutePathForWebPageFileName = "/data4/team/weijiang/human_judge_web_pages_plain_text_only_words/en0005/clueweb09-en0005-76-03988_17952_plain_text.txt"
        # queryID = "1"
        # trecID = "clueweb09-en0005-76-03988"
        # tempDebugDocumentTuple = (queryID,trecID,absolutePathForWebPageFileName)
        # debugDocumentList.append(tempDebugDocumentTuple)        
        
        
        for index,tempDebugDocumentTuple in enumerate(debugDocumentList):
            (queryID,trecID,absolutePathForWebPageFileName) = tempDebugDocumentTuple
            print "--->",index,absolutePathForWebPageFileName
            documentFileHandler = open(absolutePathForWebPageFileName,"r")
            doc_content = documentFileHandler.read()
            doc_words = doc_content.strip().split(" ")
        
            # debug
            debugVarianceList = [(0,"obama_family")]
            for tempTestTuple in debugVarianceList:
                (queryVarianceID,queryVarianceContent) = tempTestTuple
                computeComplexFeaturesSupportedData(doc_words,queryVarianceID,queryVarianceContent,outputFileHandler,cf_dicts[queryVarianceContent],df_dicts[queryVarianceContent],queryID,trecID)
                print "        cf_dicts[",queryVarianceContent,"]:",cf_dicts[queryVarianceContent]
                print "        df_dicts[",queryVarianceContent,"]:",df_dicts[queryVarianceContent]
            
            print
        
        #################################################debug ends.
        '''
        
        # finally, dump the cf_dicts and df_dicts into the disk and store as a file
        # print "cf_dicts:",cf_dicts
        # print "df_dicts:",df_dicts
        
        outputFileNameForCf_dicts = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-complexFeaturesSupportedFileForCf_dicts.txt"
        outputFileHandlerForCf_dicts = open(outputFileNameForCf_dicts,"w")
        
        for queryVarianceTuple in queryVarianceList:
            (queryVarianceID,queryVarianceContent) = queryVarianceTuple
            outputFileHandlerForCf_dicts.write(str(queryVarianceID) + " " + queryVarianceContent + " " + str(cf_dicts[queryVarianceContent]["cf_M_1"]) + " " + str(cf_dicts[queryVarianceContent]["cf_M_4"]) + " " + str(cf_dicts[queryVarianceContent]["cf_M_8"]) + " " + str(cf_dicts[queryVarianceContent]["cf_M_32"]) + " " + str(cf_dicts[queryVarianceContent]["cf_N_1"]) + " " + str(cf_dicts[queryVarianceContent]["cf_N_4"]) + " " + str(cf_dicts[queryVarianceContent]["cf_N_8"]) + " " + str(cf_dicts[queryVarianceContent]["cf_N_32"]) + "\n")
        
        outputFileHandlerForCf_dicts.close()

        outputFileNameForDf_dicts = "/data5/team/weijiang/the_new_trip_of_feature_generation/final-complexFeaturesSupportedFileForDf_dicts.txt"
        outputFileHandlerForDf_dicts = open(outputFileNameForDf_dicts,"w")
        
        for queryVarianceTuple in queryVarianceList:
            (queryVarianceID,queryVarianceContent) = queryVarianceTuple
            outputFileHandlerForDf_dicts.write(str(queryVarianceID) + " " + queryVarianceContent + " " + str(df_dicts[queryVarianceContent]["df_M_1"]) + " " + str(df_dicts[queryVarianceContent]["df_M_4"]) + " " + str(df_dicts[queryVarianceContent]["df_M_8"]) + " " + str(df_dicts[queryVarianceContent]["df_M_32"]) + " " + str(df_dicts[queryVarianceContent]["df_N_1"]) + " " + str(df_dicts[queryVarianceContent]["df_N_4"]) + " " + str(df_dicts[queryVarianceContent]["df_N_8"]) + " " + str(df_dicts[queryVarianceContent]["df_N_32"]) + "\n")
        
        outputFileHandlerForCf_dicts.close()
                
        
        
        # production
        '''
        for queryVariationTuple in queryVariationTupleList:
            (queryID,attr,queryVariations) = queryVariationTuple
            for queryVariant in queryVariations:
                computeComplexFeaturesSupportedData(doc_words,queryVariant,outputFileHandler,current_cf_dict,current_df_dict,queryID,trecID)
                print "current_cf_dict:",current_cf_dict
                print "current_df_dict:",current_df_dict        
        '''
    
